{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "uuid": "f9db1dc0-60de-4114-ac7b-caba1b3a4e63"
   },
   "source": [
    "# 使用EasyTransfer快速搭建天池大赛Baseline\n",
    "\n",
    "\n",
    "## （一）定义配置文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "uuid": "2f687623-98d6-4394-b8ce-558070dc8a6d"
   },
   "outputs": [],
   "source": [
    "config_json = {\n",
    "        \"worker_hosts\": \"localhost\",\n",
    "        \"task_index\": 1,\n",
    "        \"job_name\": \"chief\",\n",
    "        \"num_gpus\": 1,\n",
    "        \"num_workers\": 1,\n",
    "        \"preprocess_config\": {\n",
    "            \"input_schema\": \"input_ids:int:128,input_mask:int:128,segment_ids:int:128,label_id:int:1\",\n",
    "            \"sequence_length\": 128\n",
    "        },\n",
    "    \n",
    "        \"model_config\": {\n",
    "            \"pretrain_model_name_or_path\": \"pai-bert-tiny-zh\",\n",
    "        },\n",
    "        \n",
    "        \"train_config\": {\n",
    "            \"train_input_fp\": \"./data/train.list_tfrecord\",\n",
    "            \"train_batch_size\": 2,\n",
    "            \"num_epochs\": 0.01,\n",
    "            \"model_dir\": \"model_dir\",\n",
    "            \"optimizer_config\": {\n",
    "                \"learning_rate\": 1e-5\n",
    "            }\n",
    "        },\n",
    "        \n",
    "        \"predict_config\": {\n",
    "            \"predict_checkpoint_path\": None,\n",
    "            \"predict_input_fp\": \"./data/dev.list_tfrecord\",\n",
    "            \"predict_batch_size\": 2\n",
    "        }\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "uuid": "785e9792-dd80-477b-92c0-0b54e3c94213"
   },
   "source": [
    "##  （二）定义多任务数据读取器\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/admin/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1111 22:14:40.547669 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:22: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
      "\n",
      "W1111 22:14:40.548489 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:22: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
      "\n",
      "W1111 22:14:40.548957 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:27: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "W1111 22:14:40.549345 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:28: The name tf.set_random_seed is deprecated. Please use tf.compat.v1.set_random_seed instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from easytransfer import base_model, Config, FLAGS\n",
    "from easytransfer import layers\n",
    "from easytransfer import model_zoo\n",
    "from easytransfer import preprocessors\n",
    "from easytransfer.datasets import TFRecordReader\n",
    "from easytransfer.losses import softmax_cross_entropy\n",
    "from sklearn.metrics import classification_report\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiTaskTFRecordReader(TFRecordReader):\n",
    "    def __init__(self, input_glob, batch_size, is_training=False,\n",
    "                 **kwargs):\n",
    "\n",
    "        super(MultiTaskTFRecordReader, self).__init__(input_glob, batch_size, is_training, **kwargs)\n",
    "        self.task_fps = []\n",
    "        with tf.gfile.Open(input_glob, 'r') as f:\n",
    "            for line in f:\n",
    "                line = line.strip()\n",
    "                self.task_fps.append(line)\n",
    "\n",
    "    def get_input_fn(self):\n",
    "        def input_fn():\n",
    "            num_datasets = len(self.task_fps)\n",
    "            datasets = []\n",
    "            for input_glob in self.task_fps:\n",
    "                dataset = tf.data.TFRecordDataset(input_glob)\n",
    "                dataset = self._get_data_pipeline(dataset, self._decode_tfrecord)\n",
    "                datasets.append(dataset)\n",
    "\n",
    "            choice_dataset = tf.data.Dataset.range(num_datasets).repeat()\n",
    "            return tf.data.experimental.choose_from_datasets(datasets, choice_dataset)\n",
    "\n",
    "        return input_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  （三）定义分类应用\n",
    "\n",
    "- base_model: 所有应用都需要继承的父类\n",
    "- Config：用来解析配置文件的父类\n",
    "- layers：基础组件。比如Embedding，Attention等\n",
    "- model_zoo: 管理预训练模型的组件库，通过get_pretrained_model方法可调用bert模型\n",
    "- preprocessors：管理各种应用的预处理逻辑\n",
    "- softmax_cross_entropy：用于分类任务的损失函数\n",
    "\n",
    "完整的训练/评估/预测/链路，由四个函数构成\n",
    "- build_logits: 构图\n",
    "- build_loss：定义损失函数\n",
    "- build_eval_metrics：定义评估指标\n",
    "- build_predictions：定义预测输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "10ccec35-4235-4f4a-96a9-85cced4a1eb5"
   },
   "outputs": [],
   "source": [
    "class Application(base_model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(Application, self).__init__(**kwargs)\n",
    "        self.user_defined_config = kwargs[\"user_defined_config\"]\n",
    "\n",
    "    def build_logits(self, features, mode=None):\n",
    "\n",
    "        preprocessor = preprocessors.get_preprocessor(self.pretrain_model_name_or_path,\n",
    "                                                      user_defined_config=self.user_defined_config)\n",
    "\n",
    "        model = model_zoo.get_pretrained_model(self.pretrain_model_name_or_path)\n",
    "\n",
    "        global_step = tf.train.get_or_create_global_step()\n",
    "\n",
    "        tnews_dense = layers.Dense(15,\n",
    "                     kernel_initializer=layers.get_initializer(0.02),\n",
    "                     name='tnews_dense')\n",
    "\n",
    "        ocemotion_dense = layers.Dense(7,\n",
    "                             kernel_initializer=layers.get_initializer(0.02),\n",
    "                             name='ocemotion_dense')\n",
    "\n",
    "        ocnli_dense = layers.Dense(3,\n",
    "                             kernel_initializer=layers.get_initializer(0.02),\n",
    "                             name='ocnli_dense')\n",
    "\n",
    "        input_ids, input_mask, segment_ids, label_ids = preprocessor(features)\n",
    "\n",
    "        outputs = model([input_ids, input_mask, segment_ids], mode=mode)\n",
    "        pooled_output = outputs[1]\n",
    "\n",
    "        if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "            pooled_output = tf.nn.dropout(pooled_output, keep_prob=0.9)\n",
    "\n",
    "        logits = tf.case([(tf.equal(tf.mod(global_step, 3), 0), lambda: tnews_dense(pooled_output)),\n",
    "                          (tf.equal(tf.mod(global_step, 3), 1), lambda: ocemotion_dense(pooled_output)),\n",
    "                          (tf.equal(tf.mod(global_step, 3), 2), lambda: ocnli_dense(pooled_output)),\n",
    "                          ], exclusive=True)\n",
    "\n",
    "        if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "            ret = {\n",
    "                \"tnews_logits\": tnews_dense(pooled_output),\n",
    "                \"ocemotion_logits\": ocemotion_dense(pooled_output),\n",
    "                \"ocnli_logits\": ocnli_dense(pooled_output),\n",
    "                \"label_ids\": label_ids\n",
    "            }\n",
    "            return ret\n",
    "\n",
    "        return logits, label_ids\n",
    "\n",
    "    def build_loss(self, logits, labels):\n",
    "        global_step = tf.train.get_or_create_global_step()\n",
    "        return tf.case([(tf.equal(tf.mod(global_step, 3), 0), lambda : softmax_cross_entropy(labels, 15, logits)),\n",
    "                      (tf.equal(tf.mod(global_step, 3), 1), lambda : softmax_cross_entropy(labels, 7, logits)),\n",
    "                      (tf.equal(tf.mod(global_step, 3), 2), lambda : softmax_cross_entropy(labels, 3, logits))\n",
    "                      ], exclusive=True)\n",
    "\n",
    "    def build_predictions(self, output):\n",
    "        tnews_logits = output['tnews_logits']\n",
    "        ocemotion_logits = output['ocemotion_logits']\n",
    "        ocnli_logits = output['ocnli_logits']\n",
    "\n",
    "        tnews_predictions = tf.argmax(tnews_logits, axis=-1, output_type=tf.int32)\n",
    "        ocemotion_predictions = tf.argmax(ocemotion_logits, axis=-1, output_type=tf.int32)\n",
    "        ocnli_predictions = tf.argmax(ocnli_logits, axis=-1, output_type=tf.int32)\n",
    "\n",
    "        ret_dict = {\n",
    "            \"tnews_predictions\": tnews_predictions,\n",
    "            \"ocemotion_predictions\": ocemotion_predictions,\n",
    "            \"ocnli_predictions\": ocnli_predictions,\n",
    "            \"label_ids\": output['label_ids']\n",
    "        }\n",
    "        return ret_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "uuid": "c31301bb-0dc0-4506-9c94-73d76942c782"
   },
   "source": [
    "# (四）启动训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "ec5ad940-9410-4d95-8a17-788c826f18eb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1111 22:14:41.860600 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:62: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "I1111 22:14:41.861164 139963974477632 model.py:62] ***************** modelZooBasePath /home/admin/.eztransfer_modelzoo ***************\n",
      "W1111 22:14:41.861647 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:740: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "I1111 22:14:41.865174 139963974477632 model.py:747] Reading 0 files\n",
      "W1111 22:14:41.865653 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:749: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n",
      "I1111 22:14:42.109838 139963974477632 model.py:771] total number of training examples 135525\n",
      "I1111 22:14:42.110572 139963974477632 model.py:246] ***********Running in train mode***********\n",
      "I1111 22:14:42.110985 139963974477632 model.py:255] ***********Disable Tao***********\n",
      "I1111 22:14:42.111428 139963974477632 model.py:277] ***********NCCL_IB_DISABLE 0***********\n",
      "I1111 22:14:42.111767 139963974477632 model.py:278] ***********NCCL_P2P_DISABLE 0***********\n",
      "I1111 22:14:42.112101 139963974477632 model.py:279] ***********NCCL_SHM_DISABLE 0***********\n",
      "I1111 22:14:42.112422 139963974477632 model.py:280] ***********NCCL_MAX_NRINGS 4***********\n",
      "I1111 22:14:42.112729 139963974477632 model.py:281] ***********NCCL_MIN_NRINGS 2***********\n",
      "I1111 22:14:42.113036 139963974477632 model.py:282] ***********NCCL_LAUNCH_MODE PARALLEL***********\n",
      "I1111 22:14:42.113362 139963974477632 model.py:283] ***********TF_JIT_PROFILING False***********\n",
      "I1111 22:14:42.113675 139963974477632 model.py:284] ***********PAI_ENABLE_HLO_DUMPER False***********\n",
      "I1111 22:14:42.114027 139963974477632 model.py:384] ***********Single worker, Single gpu, Don't use distribution strategy***********\n",
      "I1111 22:14:42.114339 139963974477632 model.py:414] model_dir: model_dir\n",
      "I1111 22:14:42.114655 139963974477632 model.py:415] num workers: 1\n",
      "I1111 22:14:42.114956 139963974477632 model.py:416] num gpus: 1\n",
      "I1111 22:14:42.115273 139963974477632 model.py:417] learning rate: 1e-05\n",
      "I1111 22:14:42.115583 139963974477632 model.py:418] train batch size: 2\n",
      "I1111 22:14:42.115887 139963974477632 model.py:419] global batch size: 2\n",
      "I1111 22:14:42.116196 139963974477632 model.py:420] num accumulated batches: 1\n",
      "I1111 22:14:42.116508 139963974477632 model.py:421] num model replica: 1\n",
      "I1111 22:14:42.116801 139963974477632 model.py:422] num train examples per epoch: 135525\n",
      "I1111 22:14:42.117116 139963974477632 model.py:423] num epochs: 0.01\n",
      "I1111 22:14:42.117423 139963974477632 model.py:424] train steps: 678\n",
      "I1111 22:14:42.117739 139963974477632 model.py:425] save steps: 67762\n",
      "I1111 22:14:42.118072 139963974477632 model.py:426] throttle secs: 100\n",
      "I1111 22:14:42.118380 139963974477632 model.py:427] keep checkpoint max: 10\n",
      "I1111 22:14:42.118684 139963974477632 model.py:428] warmup ratio: 0.1\n",
      "I1111 22:14:42.118985 139963974477632 model.py:429] gradient clip: True\n",
      "I1111 22:14:42.119289 139963974477632 model.py:430] clip norm value: 1.0\n",
      "I1111 22:14:42.119610 139963974477632 model.py:431] log step count steps: 100\n",
      "W1111 22:14:42.119966 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:492: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "W1111 22:14:42.120330 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:497: The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions instead.\n",
      "\n",
      "I1111 22:14:42.121168 139963974477632 estimator.py:209] Using config: {'_model_dir': 'model_dir', '_tf_random_seed': 123123, '_save_summary_steps': 100, '_save_checkpoints_steps': 67762, '_save_checkpoints_secs': None, '_session_config': intra_op_parallelism_threads: 1024\n",
      "inter_op_parallelism_threads: 1024\n",
      "gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "  allow_growth: true\n",
      "  force_gpu_compatible: true\n",
      "}\n",
      "allow_soft_placement: true\n",
      ", '_keep_checkpoint_max': 10, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4b281d17f0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "W1111 22:14:42.122250 139963974477632 model_fn.py:630] Estimator's model_fn (<function EzTransEstimator._build_model_fn.<locals>.model_fn at 0x7f4b281cfb70>) includes params argument, but params are not passed to Estimator.\n",
      "I1111 22:14:42.123196 139963974477632 reader.py:78] num_parallel_batches 1\n",
      "I1111 22:14:42.123563 139963974477632 reader.py:79] shuffle_buffer_size None\n",
      "I1111 22:14:42.123873 139963974477632 reader.py:80] prefetch_buffer_size 1\n",
      "I1111 22:14:42.124194 139963974477632 reader.py:81] batch_size 2\n",
      "I1111 22:14:42.124503 139963974477632 reader.py:82] distribution_strategy None\n",
      "I1111 22:14:42.124810 139963974477632 reader.py:83] num_micro_batches 1\n",
      "I1111 22:14:42.125118 139963974477632 reader.py:84] input_schema input_ids:int:128,input_mask:int:128,segment_ids:int:128,label_id:int:1\n",
      "I1111 22:14:42.129706 139963974477632 tfrecord_reader.py:59] Reading 0 files\n",
      "I1111 22:14:42.368194 139963974477632 tfrecord_reader.py:63] ./data/train.list_tfrecord, total number of training examples 135525\n",
      "W1111 22:14:42.416970 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
      "I1111 22:14:42.441215 139963974477632 reader.py:89] Random shuffle on the whole 135525 training examples\n",
      "W1111 22:14:42.447071 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/data/util/random_seed.py:58: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W1111 22:14:42.448966 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/datasets/reader.py:104: map_and_batch (from tensorflow.python.data.experimental.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.map(map_func, num_parallel_calls)` followed by `tf.data.Dataset.batch(batch_size, drop_remainder)`. Static tf.data optimizations will take care of using the fused implementation.\n",
      "W1111 22:14:42.450617 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/datasets/tfrecord_reader.py:87: The name tf.parse_single_example is deprecated. Please use tf.io.parse_single_example instead.\n",
      "\n",
      "I1111 22:14:42.522217 139963974477632 estimator.py:1145] Calling model_fn.\n",
      "W1111 22:14:42.533562 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/preprocessors/preprocessor.py:145: The name tf.gfile.Exists is deprecated. Please use tf.io.gfile.exists instead.\n",
      "\n",
      "W1111 22:14:42.534109 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/preprocessors/tokenization.py:116: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "W1111 22:14:42.601481 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/layers/utils.py:102: The name tf.keras.initializers.TruncatedNormal is deprecated. Please use tf.compat.v1.keras.initializers.TruncatedNormal instead.\n",
      "\n",
      "W1111 22:14:42.601988 139963974477632 deprecation.py:506] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/keras/initializers.py:94: calling TruncatedNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W1111 22:14:51.277084 139963974477632 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "I1111 22:14:53.177355 139963974477632 modeling_utils.py:156] Load weights from /home/admin/.eztransfer_modelzoo/bert/pai-bert-tiny-zh/model.ckpt\n",
      "W1111 22:14:53.178091 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/model_zoo/modeling_utils.py:157: The name tf.train.init_from_checkpoint is deprecated. Please use tf.compat.v1.train.init_from_checkpoint instead.\n",
      "\n",
      "W1111 22:14:54.217112 139963974477632 deprecation.py:506] From <ipython-input-4-ab4d5f96bf6c>:33: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W1111 22:14:54.343232 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/losses/classification_regression_loss.py:22: The name tf.losses.softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.softmax_cross_entropy instead.\n",
      "\n",
      "W1111 22:14:54.491446 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/optimizers/__init__.py:39: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "W1111 22:14:54.492145 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/optimizers/__init__.py:41: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "W1111 22:14:54.496683 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py:409: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "I1111 22:14:54.500868 139963974477632 __init__.py:52] *******Warmup 67 steps***********\n",
      "I1111 22:14:54.507998 139963974477632 __init__.py:79] *******Using adam optimizer************\n",
      "W1111 22:14:54.508452 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/optimizers/__init__.py:80: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "I1111 22:14:55.556673 139963974477632 __init__.py:103] *******Num of trainable variables 3224867************\n",
      "I1111 22:14:55.557478 139963974477632 __init__.py:106] *******Clip Gradients************\n",
      "I1111 22:14:55.557855 139963974477632 __init__.py:107] *******Clip Norm Value 1.0*********\n",
      "I1111 22:14:55.642448 139963974477632 __init__.py:112] *********Num towers is 1 *********\n",
      "W1111 22:14:56.222313 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/optimizers/__init__.py:132: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n",
      "W1111 22:14:56.224287 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/utils/hooks.py:20: The name tf.train.SessionRunHook is deprecated. Please use tf.estimator.SessionRunHook instead.\n",
      "\n",
      "W1111 22:14:56.224805 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/utils/hooks.py:27: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
      "\n",
      "W1111 22:14:56.225701 139963974477632 deprecation_wrapper.py:119] From /home/admin/.local/lib/python3.6/site-packages/easytransfer/engines/model.py:554: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
      "\n",
      "I1111 22:14:56.226991 139963974477632 estimator.py:1147] Done calling model_fn.\n",
      "I1111 22:14:56.228651 139963974477632 basic_session_run_hooks.py:541] Create CheckpointSaverHook.\n",
      "I1111 22:14:57.016006 139963974477632 monitored_session.py:240] Graph was finalized.\n",
      "W1111 22:14:57.155254 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "I1111 22:14:57.162078 139963974477632 saver.py:1280] Restoring parameters from model_dir/model.ckpt-0\n",
      "W1111 22:14:57.766013 139963974477632 deprecation.py:323] From /home/admin/.local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1066: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "I1111 22:14:57.967705 139963974477632 session_manager.py:500] Running local_init_op.\n",
      "I1111 22:14:58.010382 139963974477632 session_manager.py:502] Done running local_init_op.\n",
      "I1111 22:14:59.844251 139963974477632 basic_session_run_hooks.py:606] Saving checkpoints for 0 into model_dir/model.ckpt.\n",
      "I1111 22:15:02.512282 139963974477632 basic_session_run_hooks.py:262] loss = 2.763852, step = 1\n",
      "I1111 22:15:16.617555 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 7.08944\n",
      "I1111 22:15:16.618896 139963974477632 basic_session_run_hooks.py:260] loss = 1.8116924, step = 101 (14.107 sec)\n",
      "I1111 22:15:28.402102 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 8.48569\n",
      "I1111 22:15:28.403470 139963974477632 basic_session_run_hooks.py:260] loss = 1.230713, step = 201 (11.785 sec)\n",
      "I1111 22:15:40.303261 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 8.40252\n",
      "I1111 22:15:40.304564 139963974477632 basic_session_run_hooks.py:260] loss = 2.6100245, step = 301 (11.901 sec)\n",
      "I1111 22:15:51.913907 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 8.61278\n",
      "I1111 22:15:51.915229 139963974477632 basic_session_run_hooks.py:260] loss = 1.8682294, step = 401 (11.611 sec)\n",
      "I1111 22:16:04.109765 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 8.1995\n",
      "I1111 22:16:04.111045 139963974477632 basic_session_run_hooks.py:260] loss = 1.1494207, step = 501 (12.196 sec)\n",
      "I1111 22:16:15.606740 139963974477632 basic_session_run_hooks.py:692] global_step/sec: 8.69796\n",
      "I1111 22:16:15.608087 139963974477632 basic_session_run_hooks.py:260] loss = 2.7215586, step = 601 (11.497 sec)\n",
      "I1111 22:16:25.305510 139963974477632 basic_session_run_hooks.py:606] Saving checkpoints for 678 into model_dir/model.ckpt.\n",
      "I1111 22:16:26.333187 139963974477632 estimator.py:368] Loss for final step: 1.2188876.\n"
     ]
    }
   ],
   "source": [
    "config = Config(mode=\"train\", config_json=config_json)\n",
    "app = Application(user_defined_config=config)\n",
    "\n",
    "train_reader = MultiTaskTFRecordReader(input_glob=app.train_input_fp,\n",
    "                                           is_training=True,\n",
    "                                           input_schema=app.input_schema,\n",
    "                                           batch_size=app.train_batch_size)\n",
    "\n",
    "app.run_train(reader=train_reader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# (五）启动评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I1111 22:16:32.846387 139963974477632 model.py:62] ***************** modelZooBasePath /home/admin/.eztransfer_modelzoo ***************\n",
      "I1111 22:16:32.851982 139963974477632 model.py:807] total number of predicting examples 3\n",
      "I1111 22:16:32.852515 139963974477632 model.py:469] ***********Running in predict mode***********\n",
      "W1111 22:16:32.853458 139963974477632 estimator.py:1811] Using temporary folder as model directory: /tmp/tmpesar06x9\n",
      "I1111 22:16:32.854016 139963974477632 estimator.py:209] Using config: {'_model_dir': '/tmp/tmpesar06x9', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': intra_op_parallelism_threads: 1024\n",
      "inter_op_parallelism_threads: 1024\n",
      "gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "  allow_growth: true\n",
      "  force_gpu_compatible: true\n",
      "}\n",
      "allow_soft_placement: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4b1523e710>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "W1111 22:16:32.854488 139963974477632 model_fn.py:630] Estimator's model_fn (<function EzTransEstimator._build_model_fn.<locals>.model_fn at 0x7f4b1599ce18>) includes params argument, but params are not passed to Estimator.\n",
      "I1111 22:16:32.855013 139963974477632 reader.py:78] num_parallel_batches 1\n",
      "I1111 22:16:32.855383 139963974477632 reader.py:79] shuffle_buffer_size None\n",
      "I1111 22:16:32.855694 139963974477632 reader.py:80] prefetch_buffer_size 1\n",
      "I1111 22:16:32.855990 139963974477632 reader.py:81] batch_size 2\n",
      "I1111 22:16:32.856314 139963974477632 reader.py:82] distribution_strategy None\n",
      "I1111 22:16:32.856685 139963974477632 reader.py:83] num_micro_batches 1\n",
      "I1111 22:16:32.857063 139963974477632 reader.py:84] input_schema input_ids:int:128,input_mask:int:128,segment_ids:int:128,label_id:int:1\n",
      "I1111 22:16:32.871983 139963974477632 <ipython-input-6-557019691855>:20] checkpoint_path is model_dir/model.ckpt-0\n",
      "I1111 22:16:32.942761 139963974477632 estimator.py:1145] Calling model_fn.\n",
      "I1111 22:16:33.750272 139963974477632 modeling_utils.py:156] Load weights from /home/admin/.eztransfer_modelzoo/bert/pai-bert-tiny-zh/model.ckpt\n",
      "I1111 22:16:34.495724 139963974477632 estimator.py:1147] Done calling model_fn.\n",
      "I1111 22:16:34.642759 139963974477632 monitored_session.py:240] Graph was finalized.\n",
      "I1111 22:16:34.648996 139963974477632 saver.py:1280] Restoring parameters from model_dir/model.ckpt-0\n",
      "I1111 22:16:34.946957 139963974477632 session_manager.py:500] Running local_init_op.\n",
      "I1111 22:16:34.966883 139963974477632 session_manager.py:502] Done running local_init_op.\n",
      "/home/admin/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "I1111 22:16:36.138573 139963974477632 <ipython-input-6-557019691855>:20] checkpoint_path is model_dir/model.ckpt-678\n",
      "I1111 22:16:36.210490 139963974477632 estimator.py:1145] Calling model_fn.\n",
      "I1111 22:16:36.941732 139963974477632 modeling_utils.py:156] Load weights from /home/admin/.eztransfer_modelzoo/bert/pai-bert-tiny-zh/model.ckpt\n",
      "I1111 22:16:37.703608 139963974477632 estimator.py:1147] Done calling model_fn.\n",
      "I1111 22:16:37.852930 139963974477632 monitored_session.py:240] Graph was finalized.\n",
      "I1111 22:16:37.856667 139963974477632 saver.py:1280] Restoring parameters from model_dir/model.ckpt-678\n",
      "I1111 22:16:38.329453 139963974477632 session_manager.py:500] Running local_init_op.\n",
      "I1111 22:16:38.349620 139963974477632 session_manager.py:502] Done running local_init_op.\n",
      "/home/admin/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "I1111 22:16:39.444069 139963974477632 <ipython-input-6-557019691855>:59] best ckpt 678, best best_macro_f1 0.14746666666666666\n"
     ]
    }
   ],
   "source": [
    "config = Config(mode=\"predict\", config_json=config_json)\n",
    "app = Application(user_defined_config=config)\n",
    "    \n",
    "predict_reader = MultiTaskTFRecordReader(input_glob=app.predict_input_fp,\n",
    "                                           is_training=False,\n",
    "                                           input_schema=app.input_schema,\n",
    "                                           batch_size=app.predict_batch_size)\n",
    "\n",
    "ckpts = set()\n",
    "with tf.gfile.GFile(os.path.join(app.config.model_dir, \"checkpoint\"), mode='r') as reader:\n",
    "    for line in reader:\n",
    "        line = line.strip()\n",
    "        line = line.replace(\"oss://\", \"\")\n",
    "        ckpts.add(int(line.split(\":\")[1].strip().replace(\"\\\"\", \"\").split(\"/\")[-1].replace(\"model.ckpt-\", \"\")))\n",
    "\n",
    "best_macro_f1 = 0\n",
    "best_ckpt = None\n",
    "for ckpt in sorted(ckpts):\n",
    "    checkpoint_path = os.path.join(app.config.model_dir, \"model.ckpt-\" + str(ckpt))\n",
    "    tf.logging.info(\"checkpoint_path is {}\".format(checkpoint_path))\n",
    "    all_tnews_preds = []\n",
    "    all_tnews_gts = []\n",
    "    all_ocemotion_preds = []\n",
    "    all_ocemotion_gts = []\n",
    "    all_ocnli_preds = []\n",
    "    all_ocnli_gts = []\n",
    "    for i, output in enumerate(app.run_predict(reader=predict_reader, checkpoint_path=checkpoint_path)):\n",
    "        label_ids = np.squeeze(output['label_ids'])\n",
    "        if i%3 ==0:\n",
    "            tnews_predictions = output['tnews_predictions']\n",
    "            all_tnews_preds.extend(tnews_predictions.tolist())\n",
    "            all_tnews_gts.extend(label_ids.tolist())\n",
    "        elif i%3==1:\n",
    "            ocemotion_predictions = output['ocemotion_predictions']\n",
    "            all_ocemotion_preds.extend(ocemotion_predictions.tolist())\n",
    "            all_ocemotion_gts.extend(label_ids.tolist())\n",
    "        elif i%3==2:\n",
    "            ocnli_predictions = output['ocnli_predictions']\n",
    "            all_ocnli_preds.extend(ocnli_predictions.tolist())\n",
    "            all_ocnli_gts.extend(label_ids.tolist())\n",
    "\n",
    "        if i == 20:\n",
    "            break\n",
    "\n",
    "    tnews_report = classification_report(all_tnews_gts, all_tnews_preds, digits=4)\n",
    "    tnews_macro_avg_f1 = float(tnews_report.split()[-8])\n",
    "\n",
    "    ocemotion_report = classification_report(all_ocemotion_gts, all_ocemotion_preds, digits=4)\n",
    "    ocemotion_macro_avg_f1 = float(ocemotion_report.split()[-8])\n",
    "\n",
    "    ocnli_report = classification_report(all_ocnli_gts, all_ocnli_preds, digits=4)\n",
    "    ocnli_macro_avg_f1 = float(ocnli_report.split()[-8])\n",
    "\n",
    "    macro_f1 = (tnews_macro_avg_f1 + ocemotion_macro_avg_f1 + ocnli_macro_avg_f1)/3.0\n",
    "    if macro_f1 >= best_macro_f1:\n",
    "        best_macro_f1 = macro_f1\n",
    "        best_ckpt = ckpt\n",
    "\n",
    "tf.logging.info(\"best ckpt {}, best best_macro_f1 {}\".format(best_ckpt, best_macro_f1))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
